package com.zhou.windpdfdownload;

import org.apache.poi.hssf.usermodel.*;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.CellType;
import org.jsoup.Jsoup;
import org.junit.Test;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.FileCacheQueueScheduler;
import us.codecraft.webmagic.selector.Html;

import java.io.*;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @Author: zhouweixin
 * @Description:
 * @Date: Created in 0:40 2018/10/13
 * @Modified By:
 */
public class WindProcessor implements PageProcessor {
    private Map<String, String> url2title = new HashMap<>();
    private Set<String> urls = null;
    private Pattern pattern = Pattern.compile("HYPERLINK\\(\"(.*)\",");
    private Site site = Site.me()
            .setTimeOut(5000)
            .setRetryTimes(3)
            .setCycleRetryTimes(2);

    @Override
    public void process(Page page) {
        System.out.println();

        if (page.getUrl().toString().contains("bulletin.php")) {
            //            String text = Jsoup.parse(page.getHtml().toString()).select("div#contents").text();
            try {
                String t = Jsoup.parse(page.getHtml().toString()).select("div#snapTitle").text();
                t = t.replaceAll("[\\\\/:\"<>|*]", "");
                String text = Jsoup.parse(page.getHtml().toString()).text();
                saveFile(text, t + ".txt");
            }catch (Exception e){
                e.printStackTrace();
            }

            // pdf下载地址
            String a = page.getHtml().xpath("//a[@class='big']").get();
            String href = new Html(a).xpath("//a/@href").get();
            String title = "(" + href.replace("getatt.php?", "").replace("&", "-").replace("att_id=", "").replace("id=", "") + ")" + new Html(a).xpath("//a/text()").get();
            title = title.replaceAll("[\\\\/:\"<>|*]", "");
            System.out.println("=========================");
            System.out.println(title);

            href = "http://snap.windin.com/ns/" + href;
            url2title.put(href, title);
            page.putField("url", href);
            page.putField("title", title);
            page.addTargetRequest(href);


        } else if (page.getUrl().toString().contains("getatt.php")) {
            String url = page.getUrl().toString();

            try {
                FileOutputStream fos = new FileOutputStream("output/pdf/" + MyAccessDataBase.getTitleByUrl(url));
                fos.write(page.getBytes());
                fos.close();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }

    /**
     * 保存文本文件
     *
     * @param text
     * @param title
     * @throws IOException
     */
    private void saveFile(String text, String title) {
        if (text != null && title != null) {
            String name = title.replace(".pdf", ".txt");
            try {
                File file = new File("output/txt/" + name);
                FileOutputStream fos = new FileOutputStream(file);
                fos.write(text.getBytes());
                fos.close();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }

    @Override
    public Site getSite() {
        return site;
    }

    /**
     * 主函数
     *
     * @param args
     * @throws Exception
     */
    public static void main(String[] args) throws Exception {
        new File("./output/txt/").mkdirs();
        new File("./output/pdf/").mkdirs();

        // 校验文件
        if (!new File("output/data.accdb").exists()) {
            System.out.println("下载失败：【output/data.accdb】文件不存在");
            return;
        }

        if (!new File("output/公司公告.xls").exists()) {
            System.out.println("下载失败：【output/公司公告.xls】文件不存在");
            return;
        }

        int thread = 5;
        try {
            thread = Integer.parseInt(args[0]);
        } catch (Exception e) {
        }

        WindProcessor windProcessor = new WindProcessor();
        windProcessor.urls = windProcessor.getAllUrl();
        windProcessor.run(thread);
        System.out.println("下载完成，总数为：" + windProcessor.urls.size());
    }

    /**
     * 启动函数
     *
     * @throws Exception
     */
    public void run(int thread) throws Exception {
        if (urls == null || urls.size() == 0) {
            System.out.println("地址为空, 请检查【output/公司公告.xls】文件'");
            return;
        }

        System.out.println("===========开始下载，总数为：" + urls.size() + "， 线程数为：" + thread + "==============");

        Spider.create(new WindProcessor())
                .addUrl(urls.toArray(new String[1]))
                .setScheduler(new FileCacheQueueScheduler("./output"))
                .addPipeline(new ConsolePipeline())
                .addPipeline(new WindPipeline())
                .thread(thread)
                .run();
    }

    @Test
    public void clearTitle(){
        // \\/:"<>|
        Pattern compile = Pattern.compile("[\\\\/:\"<>|]");
        String str = "---\\/:\"<>|*----";
        String s = str.replaceAll("[\\\\/:\"<>|*]", "");
        System.out.println(s);
    }

    /**
     * 获得所有的地址
     */
    public Set<String> getAllUrl() throws Exception {
        Set<String> urls = new HashSet<>();

        File file = new File("output/公司公告.xls");
        if (file.exists() == false) {
            System.out.println("【output/公司公告.xls】文件不存在");
            return urls;
        }

        InputStream is = new FileInputStream(file);
        HSSFWorkbook workbook = new HSSFWorkbook(is);
        HSSFSheet sheet = workbook.getSheetAt(0);
        HSSFRow row = sheet.getRow(0);
        int colNum = -1;
        for (int i = 0; i < row.getLastCellNum(); i++) {
            Cell cell = row.getCell(i);
            String str = cell.getStringCellValue();
            if (str.contains("公告标题")) {
                colNum = i;
                break;
            }
        }

        if (colNum == -1) {
            System.out.println("【output/公司公告.xls】文件里未找到【公告标题】列");
            return urls;
        }

        for (int i = 1; i <= sheet.getLastRowNum(); i++) {
            row = sheet.getRow(i);
            if (row != null) {
                HSSFCell cell = row.getCell(colNum);
                if (cell != null) {
                    CellType cellType = cell.getCellType();
                    if (cellType == CellType.FORMULA) {
                        String cellFormula = cell.getCellFormula();
                        Matcher matcher = pattern.matcher(cellFormula);
                        if (matcher.find()) {
                            urls.add(matcher.group(1));
                        }
                    } else if (cellType == CellType.STRING) {
                        HSSFHyperlink hyperlink = cell.getHyperlink();
                        if (hyperlink != null && hyperlink.getAddress() != null) {
                            urls.add(hyperlink.getAddress());
                        }
                    }
                }
            }
        }

        return urls;
    }
}
